# -*- coding: utf-8 -*-
import scrapy

import re
import logging
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class HuangshiSpider(scrapy.Spider):
    name = 'huangshi'
    allowed_domains = ['58.51.240.121']
    start_urls = ['http://58.51.240.121:8503/More_xm.aspx']
    project_li_temp = 'http://58.51.240.121:8503/More_xm.aspx?page={}'  # GET 项目列表url
    room_detail_url = 'http://58.51.240.121:8503/Pub_House.aspx?xmfwbm={}'  # GET 房号详情url

    def parse(self, response):
        """
        获取项目列表
        :param response:
        :return:
        """
        # 获取当前页数
        page_num = response.meta.get('page_num', None) if response.meta.get('page_num', None) else 1
        try:
            # 获取当前页项目列表分组
            project_tr_li = response.xpath("//b[contains(text(),'项目')]/../../following-sibling::tr")
            assert project_tr_li
        except:
            logger.error(f'第{page_num}页 项目列表获取为空')
        else:
            for project_tr in project_tr_li:
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = project_tr.xpath("./td[1]/a/text()").extract_first()  # 项目名称
                item_eg['projectName'] = item_eg["projectName"].strip() if item_eg[
                    "projectName"] else None  # 预处理，便于日志输出
                item_eg['projectUrl'] = project_tr.xpath("./td[1]/a/@href").extract_first()  # 项目url
                item_eg['projectId'] = self.get_id(item_eg['projectUrl'])  # 项目id
                item_eg['projectAddress'] = project_tr.xpath("./td[2]/text()").extract_first()  # 项目地址
                item_eg['districtName'] = project_tr.xpath("./td[3]/text()").extract_first()  # 项目区域
                item_eg['totalRoomNo'] = project_tr.xpath("./td[4]/text()").extract_first()  # 总套数
                item_eg['saleableRoomNo'] = project_tr.xpath("./td[5]/text()").extract_first()  # 可售套数
                # 获取项目详情
                if item_eg['projectUrl']:
                    yield response.follow(
                        item_eg['projectUrl'],
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=20,
                    )
                else:
                    logger.error(
                        f'第{page_num}页 {item_eg["projectName"]}项目url提取为空')

            # 获取总页数
            total_page_str = response.xpath(
                "//select[@id='NewProjectList1$AspNetPager1_input']/option[last()]/text()").extract_first()
            total_page_num = self.get_total_page(total_page_str)
            # 翻页
            for next_page_num in range(2, total_page_num + 1):
                yield scrapy.Request(
                    self.project_li_temp.format(next_page_num),
                    meta=dict(page_num=deepcopy(next_page_num)),
                    priority=10,
                )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['projectUrl'] = response.request.url
        item_eg['developerName'] = response.xpath(
            "//span[@id='ProjectInfo1_lblCorpName']/text()").extract_first()  # 开发商
        item_eg['projectUse'] = response.xpath(
            "//span[@id='ProjectInfo1_lblProjectType']/text()").extract_first()  # 项目用途/项目类型
        item_eg['commencementDate'] = response.xpath(
            "//span[@id='ProjectInfo1_lblJhkgrq']/text()").extract_first()  # 开工日期
        item_eg['coverageArea'] = response.xpath("//span[@id='ProjectInfo1_lblXmzgm']/text()").extract_first()  # 占地面积
        item_eg['completionDate'] = response.xpath(
            "//span[@id='ProjectInfo1_lblJhjfsyrq']/text()").extract_first()  # 竣工日期
        item_eg['estateTotalArea'] = response.xpath(
            "//span[@id='ProjectInfo1_lblZjzmj']/text()").extract_first()  # 建筑总面积
        item_eg['totalArea'] = response.xpath(
            "//span[@id='ProjectInfo1_lblsumArea']/text()").extract_first()  # 总面积/入网总面积
        item_eg['floorAreaRatio'] = response.xpath("//span[@id='ProjectInfo1_lblRjl']/text()").extract_first()  # 容积率
        item_eg['greeningRate'] = response.xpath("//span[@id='ProjectInfo1_lblJdl']/text()").extract_first()  # 绿化率
        item_eg['projectBrief'] = response.xpath("//span[@id='ProjectInfo1_lblxmjj']/text()").extract_first()  # 项目简介
        item_eg['totalSaleableRoomNum'] = response.xpath(
            "//span[@id='ProjectInfo1_lblCanSellNum']/text()").extract_first()  # 可售总套数
        item_eg['totalSaleableArea'] = response.xpath(
            "//span[@id='ProjectInfo1_lblCanSellArea']/text()").extract_first()  # 可售总面积
        item_eg['approvalArea'] = response.xpath(
            "//span[@id='ProjectInfo1_lblBackUpArea']/text()").extract_first()  # 批准面积/备案面积
        item_eg['approvalRoomNo'] = response.xpath(
            "//span[@id='ProjectInfo1_lblBackUpNum']/text()").extract_first()  # 批准套数/备案套数

        # 获取预售许可证分组
        permit_tr_li = response.xpath("//td[contains(text(),'预售许可')]/../following-sibling::tr")
        if permit_tr_li:
            # 遍历获取预售许可证信息
            for permit_tr in permit_tr_li:
                item_eg['preSalePermit'] = permit_tr.xpath("./td[1]/text()").extract_first()  # 预售许可证
                item_eg['preSaleBuilding'] = permit_tr.xpath("./td[2]/text()").extract_first()  # 预售楼栋
                item_eg['certDate'] = permit_tr.xpath("./td[3]/text()").extract_first()  # 发证日期
                permit_url = permit_tr.xpath("./td[4]/a/@href").extract_first()  # 预售许可证详情url
                item_eg['permitId'] = self.get_id(permit_url)  # 预售许可证id
                yield item_eg

                # 获取预售许可证详情页（楼栋列表页）
                if permit_url:
                    yield response.follow(
                        permit_url,
                        callback=self.parse_building_li,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=30,
                    )
                else:
                    logger.warning('{} {}-{} 预售许可证url获取为空'.format(item_eg['projectUrl'], item_eg["projectName"],
                                                                  item_eg['preSalePermit']))
        else:
            logger.warning(
                '{} {} 预售许可证列表获取为空'.format(item_eg['projectUrl'], item_eg['projectName']))

    def parse_building_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取楼栋列表分组
        building_tr_li = response.xpath("//td[contains(text(),'总层数')]/../following-sibling::tr")
        if building_tr_li:
            # 遍历获取楼栋信息
            for building_tr in building_tr_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']  # 项目名称
                item_bd['projectId'] = item_eg['projectId']  # 项目id
                item_bd['preSalePermit'] = item_eg['preSalePermit']  # 预售许可证
                item_bd['blockName'] = building_tr.xpath("./td[1]/text()").extract_first()  # 楼栋名字
                item_bd['blockName'] = item_bd["blockName"].strip() if item_bd["blockName"] else None  # 预处理，便于日志输出
                item_bd['floorTotalNo'] = building_tr.xpath("./td[2]/text()").extract_first()  # 总层数
                item_bd['totalRoomNo'] = building_tr.xpath("./td[3]/text()").extract_first()  # 总套数
                item_bd['saleableRoomNo'] = building_tr.xpath("./td[4]/text()").extract_first()  # 可售套数
                item_bd['buildingUrl'] = building_tr.xpath("./td[5]/a/@href").extract_first()  # 楼栋url
                item_bd['buildingId'] = self.get_id(item_bd['buildingUrl'])  # 楼栋id

                # 获取楼栋详情(房号列表)
                if item_bd['buildingUrl']:
                    yield response.follow(
                        item_bd['buildingUrl'],
                        callback=self.parse_building_detail,
                        meta=dict(item_bd=deepcopy(item_bd), ),
                        priority=40,
                    )
                else:
                    logger.warning('{} {}-{}-{} 楼栋详情url获取为空'.format(response.request.url, item_eg["projectName"],
                                                                    item_eg['preSalePermit'], item_bd['blockName']))
        else:
            logger.warning(
                '{} {}-{} 楼栋列表获取为空'.format(response.request.url, item_eg["projectName"], item_eg['preSalePermit']))

    def parse_building_detail(self, response):
        """
        获取楼栋详情
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        item_bd['buildingUrl'] = response.request.url
        item_bd['buildingTotalArea'] = response.xpath(
            "//span[@id='BuildingInfo1_lblJzmj']/text()").extract_first()  # 总建筑面积
        item_bd['buildingResidenceArea'] = response.xpath(
            "//span[@id='BuildingInfo1_lblZzmj']/text()").extract_first()  # 楼栋住宅面积
        item_bd['buildingNonResidenceArea'] = response.xpath(
            "//span[@id='BuildingInfo1_lblFzzmj']/text()").extract_first()  # 楼栋非住宅面积

        # 获取分区列表分组
        zone_td_li = response.xpath("//div[@id='BuildingInfo1_tdLPB']/table//td")
        if zone_td_li:
            unit_set = set()
            for zone_td in zone_td_li:
                building_zoning = zone_td.xpath("./b/text()").extract_first()
                # 当分区名称为单元名称
                unit_set.add(building_zoning) if building_zoning and '单元' in building_zoning else False
                # 提取分区id
                zone_td_str = zone_td.xpath("./@onclick").extract_first()
                zone_td_id = self.get_zone_id(zone_td_str)
                if zone_td_id:
                    # 获取单元列表分组
                    unit_td_li = response.xpath(f"//div[contains(@id,'{zone_td_id}')]//td[@valign='bottom']")
                    if unit_td_li:
                        for unit_td in unit_td_li:
                            unit_num = unit_td.xpath("./table/td[2]/text()").extract_first()
                            unit_set.add(unit_num) if unit_num and str(
                                unit_num).strip() and '单元' not in building_zoning else False
                            # 获取房号列表分组
                            room_str_li = unit_td.xpath("./table//a/@onclick").extract()
                            for room_str in room_str_li:
                                room_id = self.get_room_id(room_str)
                                if room_id:
                                    # 获取房号详情
                                    yield scrapy.Request(
                                        self.room_detail_url.format(room_id),
                                        callback=self.parse_room_detail,
                                        priority=50,
                                        meta=dict(item_bd=deepcopy(item_bd), room_id=deepcopy(room_id),
                                                  building_zoning=deepcopy(building_zoning), ),
                                    )
                                else:
                                    logger.error(
                                        '{}-{}-{} 房号id提取失败'.format(item_bd['projectName'], item_bd['blockName'],
                                                                   room_str))
                    else:
                        logger.warning('{} {}-{}-{} 单元列表获取为空'.format(response.request.url, item_bd['projectName'],
                                                                     item_bd['preSalePermit'], item_bd['blockName']))
                else:
                    logger.error(
                        '{}-{}-{} 分区id获取为空，msg:{}'.format(item_bd['projectName'], item_bd['preSalePermit'],
                                                          item_bd['blockName'], zone_td_str))

            # 楼栋单元列表
            item_bd['unitLi'] = list(unit_set) if unit_set else ['1']  # 没有单元时，默认设为1单元
            yield item_bd
        else:
            logger.warning(
                '{} {}-{}-{} 分区列表获取为空'.format(response.request.url, item_bd['projectName'], item_bd['preSalePermit'],
                                              item_bd['blockName']))

    def parse_room_detail(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        item_rm = FdcRoomItem()
        item_rm['projectName'] = item_bd['projectName']
        item_rm['projectId'] = item_bd['projectId']
        item_rm['blockName'] = item_bd['blockName']
        item_rm['buildingId'] = item_bd['buildingId']
        item_rm['roomUrl'] = response.request.url
        item_rm['roomId'] = response.meta['room_id']
        item_rm['buildingZoning'] = response.meta['building_zoning']
        item_rm['roomLocation'] = response.xpath("//span[@id='HouseInfo1_lblZl']/text()").extract_first()
        item_rm['roomFloor'] = response.xpath("//span[@id='HouseInfo1_lblFwlc']/text()").extract_first()
        item_rm['roomNo'] = response.xpath("//span[@id='HouseInfo1_lblFwfh']/text()").extract_first()
        item_rm['roomUse'] = response.xpath("//span[@id='HouseInfo1_lblFwlx']/text()").extract_first()
        item_rm['roomStructure'] = response.xpath("//span[@id='HouseInfo1_lblFwhx']/text()").extract_first()
        item_rm['forecastBuildArea'] = response.xpath("//span[@id='HouseInfo1_lblycfwjzmj']/text()").extract_first()
        item_rm['forecastInnerArea'] = response.xpath("//span[@id='HouseInfo1_lblycfwtnmj']/text()").extract_first()
        item_rm['forecastSharedArea'] = response.xpath("//span[@id='HouseInfo1_lblycfwftmj']/text()").extract_first()
        item_rm['saleStatus'] = response.xpath("//span[@id='HouseInfo1_lblxszt']/text()").extract()
        item_rm['orientation'] = response.xpath("//span[@id='HouseInfo1_lblCx']/text()").extract_first()
        item_rm['recordUnitPrice'] = response.xpath("//span[@id='HouseInfo1_lblnxjg']/text()").extract_first()
        yield item_rm

    @staticmethod
    def get_id(url):
        """
        提取id
        :param url:
        :return:
        """
        try:
            ret = url.split('=')[1]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def get_total_page(_str) -> int:
        """
        提取总页数
        :param _str:
        :return:
        """
        try:
            ret = int(_str)
        except:
            logger.error('总页数提取出错，设为默认值12')
            return 12
        else:
            return ret

    @staticmethod
    def get_room_id(_str: str) -> str or None:
        """
        获取房号id
        :param _str:
        :return:
        """
        regex = re.compile(r"Pub_House\.aspx\?xmfwbm=(.*)\'")
        try:
            ret = regex.findall(_str)[0]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def get_zone_id(_str: str):
        """
        获取分区id
        :param _str:
        :return:
        """
        regex = re.compile(r'\'(.*)\'')
        try:
            assert _str
            ret = regex.findall(_str)[0]
            assert ret
        except:
            return
        else:
            return ret
